import requests
from lxml import etree
import time
import datetime
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
import re
import random

# es = Elasticsearch(['127.0.0.1:9200'])
es = Elasticsearch(['219.224.134.226:9202'])
# 深交互动所
UA_LIST = ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)'
]
headers = {

    "Origin": "http://irm.cninfo.com.cn",
}

cookies = {
    "JSESSIONID": "8C3156598AACA37F3F8E153153AD8F94",
}

def get_result(index,doc_type,mode=1,sdate=None,edate=None):
    """
    mode
        1 抓取最近半年最新答复,即开始时间不得超过据当天时间半年
        2 抓取历史数据 时间范围间隔最大为三个月
    """
    if (not sdate or not edate):
        raise ValueError('日期参数不能为空')

    print(sdate,edate)
    data = {
    "condition.dateFrom": sdate,
    "condition.dateTo": edate,
    "condition.searchType": "content",
    "condition.marketType": "Z",
    "pageNo": "1",
    "pageSize": "10",
    "source": "2",
    "requestUri": "/ircs/interaction/lastRepliesForSzse.do",
    "requestMethod": "POST",
    "condition.searchRange":"0",
    "condition.status":"3",
    # "condition.keyWord" :"股份"
    }
    page = 1
    max_page = None
    headers['User-Agent'] = random.choice(UA_LIST)
    while 1:
        print(sdate,page,max_page)
        data['pageNo'] = '{}'.format(page)
        if mode == 1:
            url = 'http://irm.cninfo.com.cn/ircs/interaction/lastRepliesForSzse.do'
        if mode == 2:
            # print('**********************')
            url = 'http://irm.cninfo.com.cn/ircs/interaction/topSearchForSzse.do'
            data['condition.searchRange'] = "-1"
            data['requestUri'] = '/ircs/interaction/topSearchForSzse.do'
        headers['Referer'] = url
        # print(data)
        try:
            r = requests.post(url=url,headers=headers, data=data,cookies=cookies, timeout=5)
        except:
            headers['User-Agent'] = random.choice(UA_LIST)
            time.sleep(30)
            continue
        # print(r.text)
        tree = etree.HTML(r.text)
        ask_list = tree.xpath(
            '//div[@class="askBoxOuter clear"]//div[@class="msgBox"] | //div[@class="ask_Box clear"]//div[@class="msg_Box"]')
        answer_list = tree.xpath(
            '//div[@class="answerBoxOuter clear"]//div[@class="answerBox"] | //div[@class="answer_Box clear"]//div[@class="content_Box"]')
        count = len(ask_list)
        result = []
        for i in range(count):
            ask = ask_list[i]
            answer = answer_list[i]
            item = deal_ask_and_answer(ask, answer)
            if item['ask_text'].endswith('...') or item['answer_text'].endswith('...'):
                item = get_ask_and_answer_detail(item['id'])
            result.append({
                '_index': index,
                '_type': doc_type,
                '_id': item['id'],
                '_source': item})
            # print(item)
            # return
        if result:
            bulk(es, result)
        end = tree.xpath('//div[@align="center"]/text()')
        if end and end[0] == '暂无数据':
            print(end[0])
            break
        if not max_page:
            max_page = tree.xpath('//a[@title="已经是最后一页"]/../../*/text() | //a[text()=" > "]/../a/preceding-sibling::a[1]/text()')[-1]
            if max_page =='跳转':
                break
            else:
                max_page=int(max_page)
        if page == max_page:
            break
        # print(max_page)
        page += 1
        time.sleep(random.randint(1, 3))


def deal_ask_and_answer(ask, answer):
    item = {}
    ask_a_list = ask.xpath('.//a')
    answer_a_list = answer.xpath('.//a')
    item['ask_name'] = ask_a_list[0].xpath('./text()')[0]
    ask_time = ask.xpath('string(.//a[@class="date"] | .//div[@class="pubInfo"])').strip()
    # item['cname'] = ask_a_list[1].xpath('./text()')[0]
    item['ask_text'] = ask_a_list[2].xpath('./text()')[0].strip()
    item['id'] = re.search('questionId=(.*?)&',
                           ask_a_list[2].xpath('./@href')[0]).group(1)
    item['company'] = answer_a_list[0].xpath('./text()')[0]
    # item['uname'] = answer_a_list[1].xpath('./text()')[0]
    item['answer_text'] = answer_a_list[2].xpath('./text()')[0].strip()
    answer_time = answer.xpath('string(.//a[@class="date"])').strip()
    item['ask_ts'], item['ask_date'] = trans_date_ts(ask_time)
    item['answer_ts'], item['answer_date'] = trans_date_ts(answer_time)
    return item


def get_ask_and_answer_detail(id):
    url = 'http://irm.cninfo.com.cn/ircs/interaction/viewQuestionForSzse.do?questionId={id}&condition.replyOrderType=1&condition.searchRange=0'.format(id=id)
    headers = {
        "Referer": "http://irm.cninfo.com.cn/ircs/interaction/lastRepliesForSzse.do",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
    }
    r = requests.get(url=url, headers=headers, timeout=10)
    # print(r.text)
    tree = etree.HTML(r.text)
    ask = tree.xpath('//div[@class="askBoxOuter"]//div[@class="msgBox"]')[0]
    answer = tree.xpath('//div[@class="clear answerBoxOuter"]//div[@class="answerBox"]')[0]

    item = {}
    ask_a_list = ask.xpath('.//a')
    answer_a_list = answer.xpath('.//a')
    item['ask_name'] = ask_a_list[0].xpath('./text()')[0]
    ask_time = ask.xpath('string(.//a[@class="date"] | .//div[@class="pubInfo"])').strip()
    # item['cname'] = ask_a_list[1].xpath('./text()')[0]
    item['ask_text'] = ''.join(ask_a_list[1].xpath('../text()')).strip()
    item['id'] = id
    item['company'] = answer_a_list[0].xpath('./text()')[0]
    # item['uname'] = answer_a_list[1].xpath('./text()')[0]
    item['answer_text'] = ''.join(answer_a_list[1].xpath('../text()')).replace(
        '\r\n', '').replace('\t', '').strip(' ').replace('                                    ', '')
    answer_time = answer.xpath('string(.//a[@class="date"])').strip()
    item['ask_ts'], item['ask_date'] = trans_date_ts(ask_time)
    item['answer_ts'], item['answer_date'] = trans_date_ts(answer_time)
    return item


def trans_date_ts(date):
    # 转成时间戳
    date = date.replace('年', '-').replace('月', '-').replace('日', '')
    now_ts = time.time()
    if '分钟前' in date:
        minute = int(date.rstrip('分钟前'))
        ts = now_ts - minute * 60
        date = ts2date(ts)
    elif '小时前' in date:
        hour = int(date.rstrip('小时前'))
        ts = now_ts - hour * 3600
        date = ts2date(ts)
    elif '昨天' in date:
        new_time = date.lstrip('昨天')
        date = get_before_date(1) + new_time + ":00"
    elif '今天' in date:
        new_time = date.lstrip('今天')
        date = get_before_date(0) + new_time + ":00"
    elif '刚刚' in date:
        date = ts2date(time.time())
    elif '秒前' in date:
        sec = int(date.rstrip('秒前'))
        ts = now_ts - sec
        date = ts2date(ts)
    elif date.count('-') == 1:
        date = time.strftime("%Y-", time.localtime()) + date
    # print(date)
    return date2ts(date), date[:10]


def date2ts(date):
    try:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d')))
    except:
        try:
            return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M')))
        except:
            return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M:%S')))


def ts2date(ts):
    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))

def get_before_date(date_cha,date=None):
    # date_cha为当天日期的前几天,返回前几天的日期
    # 如当天2018-7-7,date_cha=1,则返回2018-7-6
    if not date:
        today = datetime.date.today()
    else:
        today = datetime.datetime.strptime(date, '%Y-%m-%d')
    oneday = datetime.timedelta(days=date_cha)
    newday = today - oneday
    return str(newday)[:10]
def get_today():
    date = time.strftime('%Y-%m-%d',time.localtime())
    return date
if __name__ == '__main__':
    # 1 抓取最近半年所有最新答复,即时间范围不得超过半年前
    # 2 抓取历史数据 时间范围间隔最大为三个月,且必须为半年前的时间
    # today = get_today()
    date = '2014-01-02'
    while date<'2018-02-22':
        get_result(mode=2,index='info',doc_type='cn',sdate=date,edate=date)
        # get_result(mode=1,index='info',doc_type='cn',sdate=date,edate=date)
        date = get_before_date(-1,date)
        time.sleep(10)
    # get_ask_and_answer_detail(id='6246578')
    # print(get_date_cha('2018-3-3','2018-5-31'))
    # print(get_before_date(1,'2019-2-20'))